package com.xiaotu.spider.guduo.pageprocessor;

import java.math.BigDecimal;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;

import org.apache.commons.lang.StringUtils;
import org.springframework.beans.factory.annotation.Autowired;
import com.xiaotu.common.exception.SpiderException;
import com.xiaotu.common.exception.SpiderExceptionCode;
import com.xiaotu.common.model.GuDuoExceptionRecordModel;
import com.xiaotu.common.model.GuDuoPublicOpinionDataModel;
import com.xiaotu.common.model.GuDuoRecordDataModel;
import com.xiaotu.common.model.TvInfoSpiderTempModel;
import com.xiaotu.common.util.CacheHandler;
import com.xiaotu.common.util.Constants;
import com.xiaotu.common.util.DateUtils;
import com.xiaotu.common.util.GsonUtils;
import com.xiaotu.common.util.HttpUtils;
import com.xiaotu.common.util.MathUtil;
import com.xiaotu.common.util.PageUtils;
import com.xiaotu.common.util.PropertiesUtil;
import com.xiaotu.common.util.RegexUtils;
import com.xiaotu.common.util.StringUtil;
import com.xiaotu.common.util.UUIDUtils;
import com.xiaotu.server.service.guduo.SupplementGuDuoRecordService;
import com.xiaotu.spider.AUrlPageProcessor;

import us.codecraft.webmagic.Page;
/***
 * 骨朵播放量数据基类
 * @author zhang
 *
 */
public abstract class GuDuoRecordDataPageProcessor extends AUrlPageProcessor {

	
	private static String TV_INFO_SPIDER_TEMP = "TV_INFO_SPIDER_TEMP_"+DateUtils.getCurrentDate(new Date(), DateUtils.FILE_NOMENCLATURE_FORMAT);
	
	@Autowired
	private CacheHandler cacheHandler;
	
	private static Map<String, String> headers = null;
	
	//用于去重剧
	private static Map<String, Object> distinctMap = new HashMap<String, Object>();
	
	//保存异常的数据
	private static List<GuDuoExceptionRecordModel> exceptionDataList = new ArrayList<GuDuoExceptionRecordModel>();
	
	@Autowired
	private SupplementGuDuoRecordService supplementGuDuoRecordService;
	
	static {
		headers = new HashMap<String,String>();
	    headers.put("Accept", PropertiesUtil.getProperty("GuDuo.site.Accept"));
	    headers.put("Accept-Encoding", PropertiesUtil.getProperty("GuDuo.site.Accept-Encoding"));
	    headers.put("Accept-Language", PropertiesUtil.getProperty("GuDuo.site.Accept-Language"));
	    headers.put("Cache-Control", PropertiesUtil.getProperty("GuDuo.site.Cache-Control"));
	    headers.put("Connection", PropertiesUtil.getProperty("GuDuo.site.Connection"));
	    headers.put("Cookie", PropertiesUtil.getProperty("GuDuo.site.Cookie"));
	    headers.put("Host", PropertiesUtil.getProperty("GuDuo.site.Host"));
	    headers.put("User-Agent", PropertiesUtil.getProperty("GuDuo.site.User-Agent"));
	}
	
	//弹幕词云url
	private String worldCloud_url = "http://ev.guduodata.com/barrage/wordcloud/%s/random";
	
	private SimpleDateFormat sdf = new SimpleDateFormat("HH:mm:ss");
	
	private String [] start_url = {"https://www.baidu.com/"};
	
	private String startDate = DateUtils.getSpecifiedDayBefore(
			DateUtils.getCurrentDate(new Date(), DateUtils.DAY_FORMAT), DateUtils.DAY_FORMAT
			);
	
	private String endDate = startDate;
	
	@Override
    public void initRunParams(Map<String, Object> map)
    {
        if(map.get("startDate") != null)
        	this.startDate = map.get("startDate").toString();
        if(map.get("endDate") != null)
        	this.endDate = map.get("endDate").toString();
    }
	
	
	
	@Override
	public void process(Page page) {
		
		Map<String, Object> extrasMap = page.getRequest().getExtras();
		
		switch (this.getPageDataType(page))
        {
            case "record_data":
                this.recordProcess(page,extrasMap);
                break;
            default:
            	this.addRequestProcess(page);
            	break;
        }
		
	}

	List<GuDuoRecordDataModel> pipeRecordList = new ArrayList<GuDuoRecordDataModel>();
	List<GuDuoPublicOpinionDataModel> pipePublicOpinionList = new ArrayList<GuDuoPublicOpinionDataModel>();
	Map<String,Object> distinctDataidMap = new HashMap<String,Object>();
	List<Map<String,Object>> wordCloudAllList = new ArrayList<Map<String,Object>>();
	@SuppressWarnings("unchecked")
	public void recordProcess(Page page,Map<String, Object> extrasMap){
		
		String json = page.getRawText();
		
		if (StringUtils.isEmpty(json))
            throw new SpiderException(SpiderExceptionCode.DataInvalideException);
        Map<String, Object> map = GsonUtils.fromJson(json, Map.class);
        if (map == null || map.isEmpty() || map.get("data") == null)
            throw new SpiderException(SpiderExceptionCode.DataInvalideException);
		List<Map<String, Object>> dataList = (List<Map<String, Object>>) map.get("data");
        if (dataList == null || dataList.isEmpty())
            throw new SpiderException(SpiderExceptionCode.DataInvalideException);
		
        List<Map<String, Object>> tempDataList = new ArrayList<Map<String, Object>>();
        
        pipeRecordList = new ArrayList<GuDuoRecordDataModel>();
        pipePublicOpinionList = new ArrayList<GuDuoPublicOpinionDataModel>();
        wordCloudAllList = new ArrayList<Map<String,Object>>();
        exceptionDataList = new ArrayList<GuDuoExceptionRecordModel>();
        
        int a = 0;//除数
		
		int b = 0;//余数
		int c = 0;//步长
		int d = 10;//批次
		
		MyThreadLocal.setVal(dataList);
		ThreadLocal<List<Map<String,Object>>> local = MyThreadLocal.local;
		if(dataList.size()%d==0){
			
			a = (int)dataList.size()/d;
			
		}else if (dataList.size()%d!=0 && dataList.size()%d>0){
			
			a = (int)dataList.size()/d;
			b = (int)dataList.size()%d;
			
		}
		int count = 0;
		for(c =0;c<=local.get().size();c+=d){
			
			try {
				Thread.sleep(1000);
			} catch (InterruptedException e) {
				e.printStackTrace();
			}
			
			if(count<a){
				tempDataList = local.get().subList(0+c, (d+c));
				System.out.println(tempDataList.size());
				specifiedStepsProcess(page,extrasMap,tempDataList);
			}
			if(count==a){
				if(b>0){
					tempDataList = local.get().subList(c, (c+b));
					System.out.println(tempDataList.size());
					specifiedStepsProcess(page,extrasMap,tempDataList);
				}
			}
			count++;
		}
		
		
		page.putField(GuDuoRecordDataModel.class.getName(), pipeRecordList);
		page.putField(GuDuoPublicOpinionDataModel.class.getName(), pipePublicOpinionList);
		page.putField("wordCloudAllList", wordCloudAllList);
		
		if(exceptionDataList.size()>0){
			supplementGuDuoRecordService.saveRecordSpiderExceptionData(exceptionDataList);
		}
		
	}
	
	//按指定步长请求接口
	@SuppressWarnings("unchecked")
	public void specifiedStepsProcess(Page page,Map<String, Object> extrasMap,List<Map<String, Object>> tempDataList){
		
		tempDataList.forEach(movieMap ->{
			
			List<Map<String,Object>> wordCloudList = new ArrayList<Map<String,Object>>();
			
         	String showId = RegexUtils.regexSearch(RegexUtils.REGEX_INTEGER, StringUtil
                     .nullToString(movieMap.get("showId")));
             if (StringUtils.isEmpty(showId))
                 return;
             
             //当前时间
             long currDate = Long.parseLong(extrasMap.get("currDate").toString());
             
             //主键
             
             String uuid = UUIDUtils.getId();
             
             //剧名
             String tvName = movieMap.get("name")==null?null:movieMap.get("name").toString();
             //平台
             String platformName = movieMap.get("platformName")==null?null:movieMap.get("platformName").toString();
             //豆瓣评分
             String doubanScore = movieMap.get("doubanScore")==null?null:movieMap.get("doubanScore").toString();
             Double doubanScoreLong = null;
             if(StringUtils.isBlank(doubanScore.replaceAll("[\u4e00-\u9fa5]", ""))){
             	doubanScore = null;
             }
             if(null!=doubanScore && StringUtils.isNotBlank(doubanScore) 
             		 && MathUtil.isNumber(doubanScore)
             		 && MathUtil.isPositiveNumber(doubanScore)){
            	 doubanScoreLong = Double.parseDouble(doubanScore);
       		 }
             //评论数
             String comment = movieMap.get("comment")==null?null:movieMap.get("comment").toString();
             Double commentLong = null;
             if(null!=comment && StringUtils.isNotBlank(comment) 
            		 && MathUtil.isNumber(comment)
            		 && MathUtil.isPositiveNumber(comment)){
            	 commentLong = Double.parseDouble(comment);
             }
             //百度指数
             String baiduIndex = movieMap.get("baiduIndex")==null?null:movieMap.get("baiduIndex").toString();
             Double baiduIndexLong = null;
             if(StringUtils.isBlank(baiduIndex.replaceAll("[\u4e00-\u9fa5]", ""))){
            	 baiduIndexLong = null;
             }else{
            	 if(null!=baiduIndex && StringUtils.isNotBlank(baiduIndex) 
                		 && MathUtil.isNumber(baiduIndex)
                		 && MathUtil.isPositiveNumber(baiduIndex)){
            		 baiduIndexLong = Double.parseDouble(baiduIndex);
                 }
             }
             //最高全网热度
             String maxNetheat = movieMap.get("gdi")==null?null:movieMap.get("gdi").toString();
             //当天播放量
             String dayPlayCount = movieMap.get("playCount")==null?null:movieMap.get("playCount").toString();
             Double dayPlayCountLong = null;
             if(null!=dayPlayCount && StringUtils.isNotBlank(dayPlayCount) 
            		 && MathUtil.isNumber(dayPlayCount)
            		 && MathUtil.isPositiveNumber(dayPlayCount)){
            	 dayPlayCountLong = Double.parseDouble(dayPlayCount);
             }
             //本平台热度
             String platFormHot = movieMap.get("hotCount")==null?null:movieMap.get("hotCount").toString();
             
             //播放平台
             String [] platFormNameArr = null;
             
             //豆瓣评分人数
             String douban_judges_count = null;
             Double douban_judges_countLong = null;
	         //豆瓣短评数
			 String doubanShortCommenNum = null;
			 Double doubanShortCommenNumLong = null;
			 //评论总数
			 String totalCommentNum = null;
			 Double totalCommentNumLong = null;
			 //总弹幕数
			 String totalBarrageNum = null;
			 Double totalBarrageNumLong = null;
			 //总新闻收录数
			 String news_count = null;
			 Double news_countLong = null;
			 //微信总公众号文章数
			 String wechat_count = null;
			 Double wechat_countLong = null;
			 //贴吧总帖子数
			 String tieba_count = null;
			 Double tieba_countLong = null;
			 //总相关微博数
			 String weibo_count = null;
			 Double weibo_countLong = null;
			 //微博总话题数
			 String weibo_tag_count = null;
			 Double weibo_tag_countLong = null;
			 //微博总阅读量
			 String weibo_tag_read_count = null;
			 Double weibo_tag_read_countLong = null;
			 //累计播放量
			 String totalPlayCount = null;
             //异常url
             String exception_url = null;
             
             try {
            	Map<String,Object> paramMap = new HashMap<String,Object>();
         		paramMap.put("tv_dataid", showId);
         		paramMap.put("resourcesiteid", Constants.SiteName.GUDUO);
         		paramMap.put("tvtype", getTvType());
            	TvInfoSpiderTempModel queryisExitsSpiderTemp = supplementGuDuoRecordService.queryisExitsSpiderTemp(paramMap);
            	if(null == queryisExitsSpiderTemp && !distinctMap.containsKey(showId)){
            		
            		exception_url = "http://ev.guduodata.com/show/simpleDetail/"+showId;
            		Thread.sleep(1000);
            		HttpUtils.TIMEOUT = 60000;
               	 	String httpGetJSONP = HttpUtils.httpGetJSONP("http://ev.guduodata.com/show/simpleDetail/"+showId,headers);
               	    Map<String, Object> everyMap = GsonUtils.fromJson(httpGetJSONP, Map.class);
               	    	List<String> platFormImgList =  (List<String>)everyMap.get("platform_img_list");
               	    	if(null != platFormImgList && platFormImgList.size()>0){
                   	    platFormNameArr = new String[platFormImgList.size()];
   	       	            for(int i=0;i<platFormImgList.size();i++){
   	       	            	if(platFormImgList.get(i).contains("youku")){
   	       	            		platFormNameArr[i] = "优酷";
   	       					}else if(platFormImgList.get(i).contains("aiqiyi")){
   	       						platFormNameArr[i] = "爱奇艺";
   	       					}else if(platFormImgList.get(i).contains("souhu")){
   	       						platFormNameArr[i] = "搜狐视频";
   	       					}else if(platFormImgList.get(i).contains("mamgguo")){
   	       						platFormNameArr[i] = "芒果TV";
   	       					}else if(platFormImgList.get(i).contains("tengxun")){
   	       						platFormNameArr[i] = "腾讯视频";
   	       					}else if(platFormImgList.get(i).contains("pjuli")){
   	       						platFormNameArr[i] = "PPTV";
   	       					}
   	       	            }
   	       	            platFormNameArr = distinctArr(platFormNameArr);
                     }
                     //将剧的基本信息存入缓存中用于匹配猫眼剧
                     Map<String,Object> guoDuoInfoMap = new HashMap<String,Object>(); 
                     //剧类型
                     String movieType = everyMap.get("type")==null?null:everyMap.get("type").toString().replaceAll("[/、]", ",").replaceAll("[ ]", "");
                     //导演
                     String director = everyMap.get("director")==null?null:everyMap.get("director").toString().replaceAll("[/、]", ",").replaceAll("[ ]", "");
                     //演员
                     String actors = everyMap.get("actors")==null?null:everyMap.get("actors").toString().replaceAll("[/、]", ",").replaceAll("[ ]", "");
                     //编剧
                     String screenwriter = everyMap.get("script_writer")==null?null:everyMap.get("script_writer").toString().replaceAll("[/、]", ",").replaceAll("[ ]", "");
                     //制片人
                     String producer = everyMap.get("producer")==null?null:everyMap.get("producer").toString().replaceAll("[/、]", ",").replaceAll("[ ]", "");
                     //出品公司
                     String issuecompany = "";
                     List<String> issuecompanyList =  (List<String>)everyMap.get("publishers");
                     if(null != issuecompanyList && issuecompanyList.size()>0){
   	       	            for(int i=0;i<issuecompanyList.size();i++){
   	       	            	issuecompany+=","+issuecompanyList.get(i);
   	       	            }
	   	       	         if(issuecompany.length()>1){
	   	       	        	 issuecompany = issuecompany.substring(1);
	   	       	         }   
                     }
                     //集数
                     String sets = everyMap.get("episode")==null?null:everyMap.get("episode").toString().replaceAll("[ ]", "");
                     if(null != sets && StringUtils.isNotBlank(sets) && sets.indexOf(".0")!=-1){
                   	  sets = sets.replace(".0", "");
                     }
                     //时长
                     String timeLength = everyMap.get("duration")==null?null:everyMap.get("duration").toString().replaceAll("[ ]", "").replace("分钟", "");
                     //上映时间
                     String relaseDate = everyMap.get("release_date")==null?null:everyMap.get("release_date").toString().replaceAll("[ ]", "");
                     //图片链接
                     String imageurl = everyMap.get("cover")==null?null:everyMap.get("cover").toString();
                     
                     guoDuoInfoMap.put("id", UUIDUtils.getId());
                     guoDuoInfoMap.put("tv_dataid", showId);
                     guoDuoInfoMap.put("tvname", tvName);
                     guoDuoInfoMap.put("imageurl", imageurl);
                     guoDuoInfoMap.put("label", movieType);
                     guoDuoInfoMap.put("director", director);
                     guoDuoInfoMap.put("actor", actors);
                     guoDuoInfoMap.put("screenwriter", screenwriter);
                     guoDuoInfoMap.put("producer", producer);
                     guoDuoInfoMap.put("issuecompany", issuecompany);
                     guoDuoInfoMap.put("sets", sets);
                     guoDuoInfoMap.put("timeLength", timeLength);
                     guoDuoInfoMap.put("tvtype", getTvType());
                     guoDuoInfoMap.put("relasedate", relaseDate);
                     guoDuoInfoMap.put("brodcastplatform", platFormNameArr);
                     guoDuoInfoMap.put("resourcesiteid", Constants.SiteName.GUDUO);
                     guoDuoInfoMap.put("collecttime", System.currentTimeMillis());
                     cacheHandler.hSet(TV_INFO_SPIDER_TEMP, showId, guoDuoInfoMap);
                     //设置失效时间
                     cacheHandler.setCacheTimeout(TV_INFO_SPIDER_TEMP);
                     
                     distinctMap.put(showId, showId);
            	}
 	 			
 	 			//骨朵舆情情况(根据dataid  进行去重)
 	            if(!(distinctDataidMap.get(showId)+"").equals(currDate)){
 	            	
 	            	exception_url = "http://ev.guduodata.com/media/simpleOverview/"+showId;
 	            	Thread.sleep(1000);
 		 			HttpUtils.TIMEOUT = 60000;
 	 				String sumJSONP = HttpUtils.httpGetJSONP("http://ev.guduodata.com/media/simpleOverview/"+showId,headers);
 	 				
 	 	 			Map<String, Object> sumMap = GsonUtils.fromJson(sumJSONP, Map.class);
 	 	 			
 	 	 			//累计播放量
 	 	 			totalPlayCount = sumMap.get("play_count_count") == null?null:sumMap.get("play_count_count").toString();
 	 	 			if(null!=totalPlayCount && StringUtils.isNotBlank(totalPlayCount)){
 	 	 				totalPlayCount = new BigDecimal(totalPlayCount).toPlainString();
 	 	 				if(!(MathUtil.isNumber(totalPlayCount)
 	 	 			    		 && MathUtil.isPositiveNumber(totalPlayCount))){
 	 	 					totalPlayCount = null;
 	 	 				}
 	 	 	        }
 	 	 			
 	 	 			//豆瓣评分人数
 	 	 			douban_judges_count = sumMap.get("douban_judges_count") == null?null:sumMap.get("douban_judges_count").toString();
 	 	 			if(null!=douban_judges_count && StringUtils.isNotBlank(douban_judges_count) 
 	 	            		 && MathUtil.isNumber(douban_judges_count)
 	 	            		 && MathUtil.isPositiveNumber(douban_judges_count)){
 	 	 				douban_judges_countLong = Double.parseDouble(douban_judges_count);
 	 	            }
 	 	 			//豆瓣短评数
 	 	 			doubanShortCommenNum = sumMap.get("douban_brief_comment_count") == null?null:sumMap.get("douban_brief_comment_count").toString();
 	 	 			if(null!=doubanShortCommenNum && StringUtils.isNotBlank(doubanShortCommenNum) 
 	 	            		 && MathUtil.isNumber(doubanShortCommenNum)
 	 	            		 && MathUtil.isPositiveNumber(doubanShortCommenNum)){
 	 	 				doubanShortCommenNumLong = Double.parseDouble(doubanShortCommenNum);
 	 	            }
 	 	 			
 	 	 			//评论总数
 	 	 			totalCommentNum = sumMap.get("comment_count") == null?null:sumMap.get("comment_count").toString();
 		            
 	 	 			if(null!=totalCommentNum && StringUtils.isNotBlank(totalCommentNum) 
 		            		 && MathUtil.isNumber(totalCommentNum)
 		            		 && MathUtil.isPositiveNumber(totalCommentNum)){
 	 	 				totalCommentNumLong = Double.parseDouble(totalCommentNum);
 		            }
 	 	 			
 	 	 			//总弹幕数
 	 	 			totalBarrageNum = sumMap.get("barrage_count") == null?null:sumMap.get("barrage_count").toString();
 	 	 			
 	 	 			if(null!=totalBarrageNum && StringUtils.isNotBlank(totalBarrageNum) 
 		            		 && MathUtil.isNumber(totalBarrageNum)
 		            		 && MathUtil.isPositiveNumber(totalBarrageNum)){
 	 	 				totalBarrageNumLong = Double.parseDouble(totalBarrageNum);
 		            }
 	 	 			
 	 	 			//总新闻收录数
 	 	 			news_count = sumMap.get("news_count") == null?null:sumMap.get("news_count").toString();
 	 	 			
 	 	 			if(null!=news_count && StringUtils.isNotBlank(news_count) 
 		            		 && MathUtil.isNumber(news_count)
 		            		 && MathUtil.isPositiveNumber(news_count)){
 	 	 				news_countLong = Double.parseDouble(news_count);
 		            }
 	 				//微信总公众号文章数
 	 	 			wechat_count = sumMap.get("wechat_count") == null?null:sumMap.get("wechat_count").toString();
 	 	 			
 	 	 			if(null!=wechat_count && StringUtils.isNotBlank(wechat_count) 
 		            		 && MathUtil.isNumber(wechat_count)
 		            		 && MathUtil.isPositiveNumber(wechat_count)){
 	 	 				wechat_countLong = Double.parseDouble(wechat_count);
 		            }
 	 				//贴吧总帖子数
 	 	 			tieba_count = sumMap.get("tieba_count") == null?null:sumMap.get("tieba_count").toString();
 	 	 			
 	 	 			if(null!=tieba_count && StringUtils.isNotBlank(tieba_count) 
 		            		 && MathUtil.isNumber(tieba_count)
 		            		 && MathUtil.isPositiveNumber(tieba_count)){
 	 	 				tieba_countLong = Double.parseDouble(tieba_count);
 		            }
 	 				//总相关微博数
 	 	 			weibo_count = sumMap.get("weibo_count") == null?null:sumMap.get("weibo_count").toString();
 	 	 			
 	 	 			if(null!=weibo_count && StringUtils.isNotBlank(weibo_count) 
 		            		 && MathUtil.isNumber(weibo_count)
 		            		 && MathUtil.isPositiveNumber(weibo_count)){
 	 	 				weibo_countLong = Double.parseDouble(weibo_count);
 		            }
 	 				//微博总话题数
 	 	 			weibo_tag_count = sumMap.get("weibo_tag_count") == null?null:sumMap.get("weibo_tag_count").toString();
 	 	 			
 	 	 			if(null!=weibo_tag_count && StringUtils.isNotBlank(weibo_tag_count) 
 		            		 && MathUtil.isNumber(weibo_tag_count)
 		            		 && MathUtil.isPositiveNumber(weibo_tag_count)){
 	 	 				weibo_tag_countLong = Double.parseDouble(weibo_tag_count);
 		            }
 	 				//微博总阅读量
 	 	 			weibo_tag_read_count = sumMap.get("weibo_tag_read_count") == null?null:sumMap.get("weibo_tag_read_count").toString();
 	 	 			
 	 	 			if(null!=weibo_tag_read_count && StringUtils.isNotBlank(weibo_tag_read_count) 
 		            		 && MathUtil.isNumber(weibo_tag_read_count)
 		            		 && MathUtil.isPositiveNumber(weibo_tag_read_count)){
 	 	 				weibo_tag_read_countLong = Double.parseDouble(weibo_tag_read_count);
 		            }
 	            	
 	 	 			//弹幕词云
 	 	 			wordCloudList = wordCloudProcess(showId,currDate);
 	 	 			
 	            	pipePublicOpinionList.add(new GuDuoPublicOpinionDataModel(
 	                		UUIDUtils.getId(), showId, currDate+"_"+platformName+"_"+tvName+"_"+showId, null, tvName,
 	                		Constants.SiteName.GUDUO, doubanScoreLong, douban_judges_countLong, doubanShortCommenNumLong,
 	                		totalCommentNumLong, totalBarrageNumLong, news_countLong, wechat_countLong, tieba_countLong,
 	            			weibo_countLong, weibo_tag_countLong, weibo_tag_read_countLong, currDate
 	                		));
 	            	distinctDataidMap.put(showId, currDate);
 	            	
 	            	if(null != wordCloudList && wordCloudList.size()>0){
 	            		wordCloudAllList.addAll(wordCloudList);
 	            	}
 	            }
 	 			
            } catch (Exception e) {
            	
				exceptionDataList.add(new GuDuoExceptionRecordModel(
						null,showId,tvName,platformName,
						DateUtils.getCurrentDate(new Date(currDate), DateUtils.SECONDS_FORMAT),
						exception_url,null,getTvType(),DateUtils.getCurrentDate(new Date(), DateUtils.SECONDS_FORMAT)));
			}
            //骨朵过程数据 
            pipeRecordList.add(new GuDuoRecordDataModel(uuid,showId,currDate+"_"+platformName+"_"+tvName+"_"+showId,tvName, platformName, doubanScore, commentLong,
            		baiduIndexLong, maxNetheat, dayPlayCountLong, platFormHot, platFormNameArr,
            		totalPlayCount, currDate)); 
                        
         });
		
	}
	
	//获取弹幕词云
	private List<Map<String,Object>> wordCloudProcess(String dataId,long currDate){
		HttpUtils.TIMEOUT = 60000;
		String worldCloudJSONP = HttpUtils.httpGetJSONP(String.format(worldCloud_url, dataId),headers);
		List<Map<String,Object>> worldCloudList = GsonUtils.fromJson(worldCloudJSONP, List.class);
		for(Map<String,Object> map : worldCloudList){
			map.put("dataid", dataId);
			map.put("resourcesiteid", Constants.SiteName.GUDUO);
			map.put("collecttime", currDate);
		}
		return worldCloudList;
	};
	
	
	//用于添加请求
	private void addRequestProcess(Page page) {
		
		String currDate = startDate;
		
		while(true){
			Map<String,Object> map = new HashMap<String,Object>();
			if(currDate.equals(endDate)){
				map.put("currDate", DateUtils.getSecondsFromDate(currDate+" "+sdf.format(new Date())));
				addUrl(page,business_url(),currDate,map);
				break;
			}else{
				map.put("currDate", DateUtils.getSecondsFromDate(currDate+" "+sdf.format(new Date())));
				addUrl(page,business_url(),currDate,map);
			}
			currDate = DateUtils.getSpecifiedDayAfterDate(currDate,DateUtils
		            .DAY_FORMAT);
		}
	}

	
	//剧类型
	protected abstract Integer getTvType();
	
	//业务请求地址
	protected abstract String [] business_url();
	
	private void addUrl(Page page,String[] urlArr,String currDate,Map<String,Object> map){
		for(String url : urlArr){
			PageUtils.addTargetRequest(page,String.format(url, currDate),map);
		}
	}
	
	
	@Override
	protected String[] getStartRequestUrls() {
		return start_url;
	}

	@Override
	protected String getSpiderType() {
		return Constants.SpiderTaskType.TV_GUDUO;
	}

	@Override
	public String getPageDataType(Page page) {
		String url = page.getRequest().getUrl();
		if(url.contains("platformId")){
			return "record_data";
		}
		
		return "addRequest";
	}
	
	
	public static class MyThreadLocal {
		
		private static ThreadLocal<List<Map<String,Object>>> local = new ThreadLocal<List<Map<String,Object>>>();
		
		public static void setVal(List<Map<String,Object>> list){
			local.set(list);
		}
		
		public static List<Map<String,Object>> getVal(){
			return local.get();
		}
		
		public static void init(){
			local.remove();
		}
		
	}
	
	
	//set集合去重，不改变原有的顺序
    public String [] distinctArr(String[] arr){
        String [] newArr = new String[]{};
        List<String> newList = new ArrayList<String>();
        Set<String> set=new HashSet<String>();
        for (String str:arr) {
            if(set.add(str)){
            	newList.add(str);
            }
        }
        newArr = newList.toArray(newArr);
		return newArr;
    }
    
}
